package com.dozhn.crawler;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.util.EntityUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;

/**
 * Created by zhangzhen on 2017/1/5.
 */
public class HighAnonymousProxy {

    public static void getProxy(String url) throws IOException{
        //
        HttpClient client = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet(url);

        httpGet.addHeader(new BasicHeader("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36"));
        httpGet.addHeader(new BasicHeader("Cookie","channelid=0; sid=1483593427977192; _gat=1; _ga=GA1.2.199317220.1483419154; Hm_lvt_7ed65b1cc4b810e9fd37959c9bb51b31=1483422363,1483422363,1483423835,1483593514; Hm_lpvt_7ed65b1cc4b810e9fd37959c9bb51b31=1483594289"));

        HttpResponse response = client.execute(httpGet);

        if (response.getStatusLine().getStatusCode()== HttpStatus.SC_OK)
        {
            HttpEntity entity = response.getEntity();
            String html = EntityUtils.toString(entity);
            //System.out.println(html);
            Document document = Jsoup.parse(html);

            Elements elements = document.select("table.table.table-bordered.table-striped");
            Elements tbElements = elements.select("tbody");
            Elements trs = tbElements.select("tr");

            for (Element tr:
                 trs) {
                String ip = tr.select("td[data-title=IP]").text();
                String port = tr.select("td[data-title=PORT]").text();
                System.out.println(ip+"="+port);
            }
        }else{
            System.out.println(response.getStatusLine());
        }


    }

    public static void main(String[] args) throws IOException{
        String index = "http://www.kuaidaili.com/";
        String nextUrl = "http://www.kuaidaili.com/proxylist/";
        //getProxy("http://www.kuaidaili.com/free/inha/2/");
        for (int i=1;i<10;i++){
            if (i!=1){
                getProxy(nextUrl+i+"/");
            }
            else
                getProxy(index);
        }
    }
}
